from scrapy import signals
 
from selenium.webdriver import Chrome
from selenium import webdriver
from scrapy.http import HtmlResponse
import time
 
class SeleniumDownloaderMiddleware(object):
 
    def process_request(self, request, spider):
        # 获取request内的url
        request_url = request.url
        # 去除非必要的日志输出
        options = webdriver.ChromeOptions()
        options.add_experimental_option("excludeSwitches", ["enable-logging"])
        webBrowser = Chrome(options=options)
        # 发起Chrome浏览器访问request内的url
        webBrowser.get(request_url)
        time.sleep(2)
        # Chrome浏览器内的源码赋值到pageSource
        pageSource = webBrowser.page_source
        webBrowser.close()
 
        # 创建响应对象,返回。
        res = HtmlResponse(url=request_url, body=pageSource,
                           encoding='utf-8', request=request)
        return res
